package jarvis.fs;

import jarvis.fs.document.DocumentVector;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.store.Directory;

/**
 * 
 * Function: The Entropy-based measure to get feature terms
 * @author Jarvis.Guo
 *
 */
public class EN extends FeatureSelection {

	/**
	 * @param dir
	 */
	public EN(Directory dir) {
		super(dir);
		
	}

	/* (non-Javadoc)
	 * @see jarvis.fs.FeatureSelection#getTopNFeature()
	 */
	@Override
	public List<ComparableTerm> getFeatures() {
		List<ComparableTerm> result = new ArrayList<ComparableTerm>();
		DocumentVector[] vectors = DocumentVector.getVectors(indexReader);
		try {
			TermEnum te = indexReader.terms();
			int termNum = 0;
			while (te.next()) {
				double value = E(termNum,vectors);//compute the score by E(t)
				result.add(new DoubleComparableTerm(te.term(),value));
				termNum++;
			}
		}
		catch(IOException ex)
		{
			ex.printStackTrace();
			throw new RuntimeException(ex);
		}
		
		return result;
	}
	
	
	private double E(int termNum,DocumentVector[] vectors)
	{
		int docCount = vectors.length;
		double result = 0.0;
		/**
		 * A half-matrix that represent the similarity between documents 
		 * after removed the term,like
		 * 1
		 * 0.4	1
		 * 0.5	0.3	1
		 * ................
		 * which sMatrix[i][j] equals the similarity of ith document and jth document
		 */
		double[][] sMatrix = sMatrix(vectors,termNum);
		
		for(int i=0;i<docCount;i++)
		{
			for(int j=0;j<docCount;j++)
			{
				if(i==j) continue;
				double s;
				if(i>=j) s = sMatrix[i][j];
				else s = sMatrix[j][i];
				//the formula
				result += s*Math.log10(s)+(1-s)*Math.log10(1-s);
			}
		}
		return -1 * result;
	}
	
	/**
	 *  A half-matrix that represent the distance between documents 
	 * after removed the term
	 * @param vectors
	 * @param termNum
	 * @return
	 */
	private double[][] distanceMatrix(DocumentVector[] vectors,int termNum)
	{
		double[][] result = new double[vectors.length][];
		for(int i=0;i<vectors.length;i++)
		{
			result[i] = new double[i+1];
		}
		double[] termTFIDF = new double[vectors.length];
		//get the tfidf score of all documents
		for(int i=0;i<vectors.length;i++)
		{
			termTFIDF[i] = vectors[i].getTermNumTFIDF(termNum);
		}
		for(int i=0;i<result.length;i++)
		{
			for(int j=0;j<=i;j++)
			{
				//same documents, distance=0
				if(i==j) result[i][j] = 0.0;
				else
				{
					//origin similarity
					double sim = vectors[i].similary(vectors[j]);
					//get the similarity after removed the term
					sim -= termTFIDF[i]*termTFIDF[j];
					sim /= Math.sqrt(1-termTFIDF[i]*termTFIDF[i]);
					sim /= Math.sqrt(1-termTFIDF[j]*termTFIDF[j]);
					//distance = 1-similarity
					result[i][j] = 1-sim;
				}
			}
		}
		
		
		return result;
	}
	
	
	/**
	 * Get the similarity half-matrix
	 * @param vectors
	 * @param termNum
	 * @return
	 */
	private double[][] sMatrix(DocumentVector[] vectors,int termNum)
	{
		//get distance matrix first
		double[][] distanceMatrix = distanceMatrix(vectors,termNum);
		//compute the average distance in all documents
		double averageDistance = 0.0;
		for(double[] dd:distanceMatrix)
		{
			for(double d:dd)
			{
				averageDistance += d;
			}
		}
		averageDistance = averageDistance*2/(vectors.length*(vectors.length-1));
		//follow the formula to get similarity
		double a = Math.log(0.5)/averageDistance * (-1);
		for(int i=0;i<distanceMatrix.length;i++)
		{
			double[] dd = distanceMatrix[i];
			for(int j=0;j<dd.length;j++)
			{
				dd[j] = Math.pow(Math.E, (-1)*a*dd[j]);
			}
		}
		return distanceMatrix;
		
	}
	
	

}
